#!/usr/local/bin/php
<?php
require_once dirname(__FILE__) . '/lib/parser.php';
require_once dirname(__FILE__) . '/../../lib/Forage.php';
require_once 'Console/Getopt.php';

$short = 'vi:s:hc:';
$long  = array('verbose', 'index=', 'source=', 'help', 'categories=');

/**
 * Faorage indexing Wikipedia parser action
 *
 * Index parsed WikipediaPage objects with a Forage object.
 */
class ForageIndexingAction implements WikipediaParserAction
{
  /**
   * @var Forage
   */
  private $forage;

  /**
   * Counter for scheduling flushes
   *
   * @var int
   */
  private $counter;

  /**
   * Class constructor
   *
   * @param Forage $forage
   */
  public function __construct(Forage $forage)
  {
    $this->forage  = $forage;
    $this->counter = 0;
  }

  /**
   * Interface method
   * Send WikpediaPage objects to Forage for indexing
   *
   * @param WikipediaPage $page
   */
  public function parsedPage(WikipediaPage $page)
  {
    $doc = new ForageDocument();
    $doc->add('uid',     $page->id, array('uid'=>true));
    $doc->add('title',   $page->title, array('boost'=>2));
    $doc->add('content', $page->text_stripped, array('stored'=>false));
    // add the categories
    foreach ($page->categories as $category) {
      $doc->add('category', $category, array('stored'=>false, 'facet'=>true));
    }
    $this->forage->add($doc);
    
    if (++$this->counter%100==0) {
      $this->forage->flush();
      $this->forage->optimize();
    }
  }
  
  /**
   * Object destructor
   * Flush all changes to the Forage engine
   */
  public function __destruct()
  {
    $this->forage->flush();
    $this->forage->optimize();
  }
}

function showHelp()
{
  $help = <<<END_TEXT
NAME
  parser - parse a Wikipedia xml dump and index it through Forage

SYNOPSIS
  php parser.php [options] [wikipedia-dump-file(s)]

DESCRIPTION
  The parser utility of the wikipedia package parses Wikipedia dump
  files and indexes them through Forage in a supported backend.

OPTIONS
  -v, --verbose
    Print verbose logging messages
  -i DSN, --index=DSN
    The DSN used to load a Forage object to index the documents with.
  -s FORMAT, --source=FORMAT
    Set the source format, 'full' or 'mini'.
  -c FILE, --categories=FILE
    File containing category patterns, one per line.
  -h, --help
    Show this help

EXAMPLE
  parser --verbose --index='solr:127.0.0.1:8080' /path/to/wikipedia/dumps-split-*

END_TEXT;
  echo $help;
}

$con  = new Console_Getopt();
$args = $con->readPHPArgv();
array_shift($args);
$options = $con->getopt2($args, $short, $long);
$verbose = false;
$dsn     = null;
$source  = null;
$files   = array();
$categories = array();

if ($options instanceof PEAR_Error) {
  echo "Pear Error: " . $options->getMessage() . "\n";
  die();
}

foreach ($options[0] as $option) {
  switch ($option[0]) {
  case '--verbose':
  case 'v':
    $verbose = true;
    break;
  case '--index':
  case 'i':
    $dsn = $option[1];
    break;
  case 'c':
  case '--categories':
    if (!file_exists($option[1]) || !is_readable($option[1])) {
      throw new Exception("Cannot read file [" . $option[1] . "]");
    }
    $categories = file($option[1], FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES);
    break;
  case 's':
  case '--source':
    $source = $option[1];
    break;
  case '--help':
  case 'h':
  default:
    showHelp();
    die();
    break;
  }
}

$files = $options[1];

if (!$files) {
  echo "ERROR: No files provided\n";
  die();
} elseif (!$dsn) {
  echo "ERROR: No index DSN set\n";
  die();
} elseif ($source!='full' && $source!='mini') {
  echo "ERROR: Unrecognised source format\n";
  die();
}

if ($verbose) {
  echo "Options parsed, setting up indexer\n";
}

$forage = Forage::create($dsn);

if ($verbose) {
  echo "Forage created\n";
}

$action = new ForageIndexingAction($forage);
if ($verbose) {
  $forage->startProfiler();
  $logger = new ConsoleLogger();
  $action = new ProgressLoggingAction($action, $logger, 'Indexed: ', 100);
  $action = new ForageLoggingAction($action, $logger, $forage, 100);
}

if ($categories) {
  $action = new CategoryPatternFilterAction($action, $categories);
}

if ($verbose) {
  $action = new ProgressLoggingAction($action, $logger, 'Parsed: ');
}

if ($source == 'full') {
  $parser = new WikipediaParser($action);
} else {
  $parser = new MyWikiParser($action);
}
echo "Parser created\n";

if ($verbose) {
  $parser->setVerbose();
}

try {
  $parser->parseFile($files);
  if ($verbose) {
    $profiler = $forage->profiler;
    echo "Profile\n";
    echo "Hits:  " . $profiler->hits('add') . "\n";
    echo "Total: " . $profiler->total('add') . "\n";
    echo "Rate:  " . $profiler->rate('add') . "\n";
  }
} catch (Exception $e) {
  echo "Error: " . $e->getMessage() . "\n";
  throw $e;
}
